# coding:utf-8
import re
import string
from collections import Counter

stop_words_list = [
                      '', ' ', '、', '＂', '·', '̋', '（', '）', '◉', '‿', '【', '】', '：', '🦄', '，', '🥊', '•',
                      '‖', '－', 'é', '「', '」',
                  ] + list(string.punctuation)

content_pattern = re.compile('([\u4e00-\u9fa5]+)')


def get_cn(target):  # 只支持中
    ret = content_pattern.findall(target)
    return ret if ret else []


def sub_useless(target):
    target = ''.join(ch for ch in target if ch not in stop_words_list)
    return target


filter_rule = lambda w: (
    w
    and not w.isdigit() and len(w) < 8
    and w not in stop_words_list
)


def filter_words(word_list, limit=0, only_cn=False):
    word_list = map(lambda d: d[0], filter(lambda d: d[1] > limit, Counter(word_list).items()))
    word_list = filter(filter_rule, map(sub_useless, word_list))

    if only_cn:
        ret = []
        for word in word_list:
            ret.extend(get_cn(word))
        return ret
    else:
        return word_list
